{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import csv\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "src_path = 'data/train.csv'\n",
    "dst_path = 'data/train_tiny.csv'\n",
    "fe_dst_path = 'data/train_tiny_new.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def sample_data(sample_pct,src,dst):\n",
    "    np.random.seed(99)\n",
    "\n",
    "    f=csv.writer(open(dst, 'wb'))\n",
    "    for i, row in enumerate(csv.reader(open(src))):\n",
    "        if i == 0:\n",
    "            f.writerow(row)\n",
    "        else:\n",
    "            if np.random.uniform(0, 1, 1) < sample_pct:\n",
    "                f.writerow(row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def check_whether_two_fe_consistent(fe1, fe2):\n",
    "    \"\"\"Determine whether two features are consistent\"\"\"\n",
    "    fe_dict = {}\n",
    "    element_num = fe1.size\n",
    "    for i in range(element_num):\n",
    "        fe1_val, fe2_val  =  fe1[i], fe2[i]\n",
    "        if fe_dict.has_key(fe1_val):\n",
    "            if fe2_val != fe_dict[fe1_val]:\n",
    "                return False\n",
    "        else:\n",
    "            fe_dict[fe1_val] = fe2_val\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#对原始数据进行下采样\n",
    "sample_data(0.01/40, src_path, dst_path)\n",
    "all_data = pd.read_csv(dst_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "app_id app_domain\n",
      "app_id app_category\n",
      "device_ip C20\n",
      "C14 C15\n",
      "C14 C16\n",
      "C14 C17\n",
      "C14 C21\n",
      "C17 C21\n"
     ]
    }
   ],
   "source": [
    "categorical_features = all_data.select_dtypes(include = [\"object\",\"int64\"]).columns\n",
    "categorical_features_num = categorical_features.size\n",
    "drop_list = []\n",
    "for i in range(categorical_features_num):\n",
    "    for j in range(i+1, categorical_features_num):\n",
    "        if(True == check_whether_two_fe_consistent(all_data[categorical_features[i]], all_data[categorical_features[j]])):\n",
    "            drop_list.append(categorical_features[j])\n",
    "            print categorical_features[i], categorical_features[j]\n",
    "\n",
    "all_data.drop(drop_list, inplace = True, axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:5: DeprecationWarning: \n",
      ".ix is deprecated. Please use\n",
      ".loc for label based indexing or\n",
      ".iloc for positional indexing\n",
      "\n",
      "See the documentation here:\n",
      "http://pandas.pydata.org/pandas-docs/stable/indexing.html#ix-indexer-is-deprecated\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "all_data[\"day_hour\"] = np.round(all_data[\"hour\"] % 100)\n",
    "all_data.drop([\"hour\"], inplace = True, axis = 1)\n",
    "\n",
    "all_data['app_id_cat'] = 0\n",
    "all_data.ix[all_data.app_id.values=='ecad2386','app_id_cat'] = 1\n",
    "\n",
    "all_data['device_id_cat'] = 0\n",
    "all_data.ix[all_data.device_id.values=='a99f214a','device_id_cat'] = 1\n",
    "\n",
    "all_data.drop(['app_id', 'device_id'], inplace = True, axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_data.to_csv(fe_dst_path, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
